Big Data Management - Exercises 1

 map(key, value):
//key: large file of integers; value: numbers in the document
   for each number n in value:
	emit(n, 1)

reduce(key, value):
//keys: a number; value: an iterator over counts
      output_number = 0
      result_count = 0
      if output_number == number:
	result_count += count
       emit(output_number)

map(key,value):
// key: file name, value: integers in the file
   for each integer i in value:
       emit (i,1)
Reduce: Turn the value list into 1.
reduce(key, value)
//key: an integer, value: an iterator over integers
   current_integer = None
   for each i in values:
      if (current_integer == i)
         #do nothing
      else
         current_integer = i
         emit(i)

#mapper.py
import sys
import io
for line in input_stream:
 line = line.strip()
 integers=line.split()
 for integer in integers:
   print('%i\t%i' % (int(integer), 1))

#reducer.py
from operator import itemgetter
import sys

current_integer = None
current_count = 0
integer = None

#input comes from STDIN
for line in sys.stdin:
  #remove leading and trailing whitespace
  line = line.strip()

  # parse the input we got from mapper.py
  integer, count = line.split('\t', 1)
  try:
     count = int(count)
  except ValueError:
  #count was not a number, so silently
  #ignore/discard this line
    continue

  # this IF-switch only works because Hadoop sorts map output
  # by key (here: integer) before it is passed to the reducer
  if current_integer == integer:
    #do nothing
  else:
    if current_integer:
    # write result to STDOUT
      print ('%s' % (int(current_integer))) #emit the unique integer
    current_count = count
    current_integer = integer

# do not forget to output the last integer if needed!
if current_integer == integer:
  print ('%s' % (int(current_integer))) #emit the unique integer